package com.yufei.infoExtractor.parallel.thread;

import it.sauronsoftware.cron4j.Task;


import it.sauronsoftware.cron4j.TaskExecutionContext;

import java.io.IOException;

import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.CountDownLatch;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import com.yufei.infoExtractor.cache.InfoExtractorCache;
import com.yufei.infoExtractor.core.HActionContext;
import com.yufei.infoExtractor.extractor.UrlExtractor;
import com.yufei.infoExtractor.extractor.fetchdecision.DecideByIsHadFetched;
import com.yufei.infoExtractor.extractor.fetchdecision.DecideByUrlSimilarity;
import com.yufei.infoExtractor.extractor.fetchdecision.DecideIsTargetLinkOrNot;
import com.yufei.infoExtractor.extractor.fetchdecision.DecisionMaker;
import com.yufei.infoExtractor.listener.InfoExtractorEvent;
import com.yufei.infoExtractor.pfw.InfoExtractorDao;
import com.yufei.infoExtractor.pfw.entity.PropertyMatch;
import com.yufei.infoExtractor.pfw.entity.Relatedlink;
import com.yufei.infoExtractor.pfw.entity.Seedsite;
import com.yufei.infoExtractor.task.InfoExtractorCommonTask;
import com.yufei.infoExtractor.util.AppUtil;
import com.yufei.infoExtractor.util.CommonUtil;
import com.yufei.infoExtractor.util.EncryptUtil;
import com.yufei.infoExtractor.util.ExceptionUtil;
import com.yufei.infoExtractor.util.FactoryUtil;
import com.yufei.infoExtractor.util.PatternUtils;

/**
 * @author jasstion
   2012-11-23
   url extractor
 *
 */
public class UrlParserThread extends Task {
	public static final Log mLog = LogFactory.getLog(UrlParserThread.class);
	public InfoExtractorDao infoExtractorDao=(InfoExtractorDao) AppUtil.getBeanFromBeanContainer(InfoExtractorDao.class);
    private HActionContext context=null;
	private List<String> readyUrls = null;
	private CountDownLatch countDownLatch = null;
    
	public UrlParserThread(ThreadLocal<HActionContext> threadLocalcontext, List<String> readyUrls,
			CountDownLatch countDownLatch) {
		super();
    this.context=threadLocalcontext.get();
    this.readyUrls = readyUrls;
		this.countDownLatch = countDownLatch;
	}

	

	public List<String> getReadyUrls() {
		return readyUrls;
	}

	public void setReadyUrls(List<String> readyUrls) {
		this.readyUrls = readyUrls;
	}

	public CountDownLatch getCountDownLatch() {
		return countDownLatch;
	}

	public void setCountDownLatch(CountDownLatch countDownLatch) {
		this.countDownLatch = countDownLatch;
	}

	
	
		
	public void extractorUrls(HActionContext context,TaskExecutionContext taskExecutionContext){
		// TODO Auto-generated method stub
	//	InfoExtractorCache< String, String> urlCache=(InfoExtractorCache<String, String>) context.get("urlCache");
		Integer currentDepth = context.getCurrentDepth();
		Integer depth = context.getSeedsite().getDepth();
		Set[] linkSet =context.getLinkSets();
		Set<String> linkFingerPrints = context.getLinkFingerPrints();
		Iterator<String> iter = readyUrls.iterator();
		String htmlContent = null;
		List<PropertyMatch> propertyMatches=context.getSeedsite().getPattern().getPropertyMatches();
		Seedsite seed=context.getSeedsite();
		String urlMd5=null;
		String  url1 = null;
		//UrlExtractor urlExtractor=(UrlExtractor) context.get(UrlExtractor.EXTRACTOR_KEY);
        //对采集到的链接进行初步处理：递归抓取，获取每个链接的内容以及保存relatedlink
		for(String url:readyUrls){
			
			taskExecutionContext.pauseIfRequested();
			 if(taskExecutionContext.isStopped()){
	        	   throw new IllegalStateException();
	           }
			urlMd5= EncryptUtil.md5(url);

			// 将此链接放入链接指纹用于防止此链接被重复处理
			linkFingerPrints.add(urlMd5);
			
			 htmlContent=getHtmlContent(url);
			if(CommonUtil.isEmptyOrNull(htmlContent)){
				continue;
			}
	
			String filterregexes = CommonUtil.getPropertyMatchFromListByPropertyName("linkFilterRegex", propertyMatches).getMatch();
				
			List<String> filterUrls = PatternUtils.getListStrByRegex(
					htmlContent, filterregexes);
			if (CommonUtil.isEmptyOrNull(filterUrls)) {
				mLog.debug("在对链接：'"+url+"'进行子链接提取的过程中，根据链接匹配正则表达式'"+filterregexes+"'对网页内容：'"+htmlContent+"'匹配失败，请检查网页内容或者是链接匹配表达式是否有误");
				continue;
			}
			int totalValidateUrlCount=0;
			for (String extractedUrl : filterUrls) {
				try {
				mLog.debug("开始对子链接:'"+extractedUrl+"'进行链接合法性检测以及链接补全");
				taskExecutionContext.pauseIfRequested();
				 if(taskExecutionContext.isStopped()){
		        	   throw new IllegalStateException();
		           }
				if (!CommonUtil.isValidUrl(extractedUrl)) {
	
					continue;
	
				}
				url1 = CommonUtil.formatUrl(extractedUrl);
				url1 = CommonUtil.normalizeUrl(extractedUrl, url);
				
		      if(url1==null){
		    	  continue;
		      }
				boolean isOk=FactoryUtil.getDecisionMakerByType(DecisionMaker.hasFetched).makeDecision(context, url1)
						&&FactoryUtil.getDecisionMakerByType(DecisionMaker.isUsefullLink).makeDecision(context, url1);
				if (!isOk) {
					
					continue;
				}
				mLog.debug("补全后的链接为'"+url1+"'");
				totalValidateUrlCount++;
			

				context.getInfoExtractorCommonTask().increaseUrlExtractedNumber();
				//如果是最后一次递归解析链接的话，并且链接是目标链接则应该直接保存
				if(FactoryUtil.getDecisionMakerByType(DecisionMaker.isTargetLink).makeDecision(context, url)){
					if(currentDepth+2>=depth){
						Class entityClass =context.getTargetEntity().getClass();
						Relatedlink tempLink=null;
						String tempHtmlContent=null;
						tempHtmlContent=getHtmlContent(url1);
						if(tempHtmlContent==null){
							continue;
						}
						
							tempLink=(Relatedlink) entityClass.newInstance();
						
						
						saveRelatedLink(context.getInfoExtractorCommonTask(), tempHtmlContent, EncryptUtil.md5(url1), url1,
								tempLink);
						
						
					}
				}
				
		
				
				

				linkSet[currentDepth + 1].add(url1);
				//加入任务中的readyUrls中
				//urlExtractor.getTask().getReadyUrls().add(url1);
				} catch (Exception e1) {
					// TODO Auto-generated catch block
					mLog.info(ExceptionUtil.getExceptionDetailsMessage(e1));
					continue;
				} 
				
	
			}
			if(FactoryUtil.getDecisionMakerByType(DecisionMaker.isTargetLink).makeDecision(context, url)&&FactoryUtil.getDecisionMakerByType(DecisionMaker.hasFetched).makeDecision(context, url1)){
				Relatedlink currentLink=(Relatedlink) context.getTargetEntity();
				
				saveRelatedLink(context.getInfoExtractorCommonTask(), htmlContent, urlMd5, url, currentLink);	
			}
			
			mLog.debug("子链接'"+url+"'总共抽取到有效链接个数为：'"+totalValidateUrlCount+"'");


		}
	}



	private void saveRelatedLink(
			InfoExtractorCommonTask infoExtractorCommonTask,
			String htmlContent, String urlMd5, String url,
			Relatedlink currentLink) {
		currentLink.setHashCode(urlMd5);
		currentLink.setLink(url);
		currentLink.setFindTime(new Date());
		currentLink.setOriginalHtmlContent(htmlContent);
		//用于表示当前的采集信息属于那个task
		currentLink.setTaskId(infoExtractorCommonTask.getTask().getId());
		currentLink.setProcessed(false);
		infoExtractorDao.saveEntity(currentLink);

	}


private String getHtmlContent(String url){
	String htmlContent=null;
	try {
		htmlContent = CommonUtil.getHtmlContent(context.getSeedsite().getProtocolType(), context.getSeedsite().isRequestExecuteJs(), url);
	   

	} catch (IOException e) {
		// TODO Auto-generated catch block
		mLog.info("在对链接进行子链接提取的过程中访问+'"+url+"'出错，直接忽略该Url的子链接提取");
		
	}
	if (CommonUtil.isEmptyOrNull(htmlContent)) {
		mLog.info("链接+'"+url+"'请求返回的内容为空，直接忽略该Url的子链接提取");

		
	}
	return htmlContent;
	
}
	@Override
	public void execute(TaskExecutionContext taskExecutionContext) throws RuntimeException {
		// TODO Auto-generated method stub
		extractorUrls(context,taskExecutionContext);

		countDownLatch.countDown();
		mLog.info("线程执行完毕");

	}
	public boolean canBePaused() {
		return true;
	}

	public boolean canBeStopped() {
		return true;
	}

	public boolean supportsCompletenessTracking() {
		return true;
	}

	public boolean supportsStatusTracking() {
		return true;
	}


}
